import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;

import org.json.JSONException;

public class DocumentSampling {
	public static HashMap<String, HashSet<String>> getAllUrl(String host,
			ArrayList<String> classcifications) throws IOException {
		HashMap<String, HashSet<String>> result = new HashMap<String, HashSet<String>>();

		for (String classcification : classcifications) {
			String[] classes = classcification.split("/");
			for (String cls : classes) {
				Node clsNode = Node.findByName(cls.toLowerCase(),
						ClassifyAlgorithm.categoryTree);
				if (clsNode.isLeaf)
					continue;

				HashSet<String> clsUrls = new HashSet<String>();
				HashSet<String> titleLines = new HashSet<String>();

				for (Node subCategory : clsNode.subCategory)
					for (String keyword : subCategory.keyWords)
						titleLines.add(host + " " + subCategory.name + " "
								+ keyword);

				FileInputStream fstream = new FileInputStream("cache.txt");
				DataInputStream in = new DataInputStream(fstream);
				BufferedReader br = new BufferedReader(
						new InputStreamReader(in));
				String strLine;
				while ((strLine = br.readLine()) != null) {
					if (titleLines.contains(strLine)) {
						String nextLine = br.readLine();
						String[] urls = nextLine.split(" ");
						for (int i = 1; i < urls.length; i++)
							clsUrls.add(urls[i]);
					}
				}
				br.close();

				result.put(cls, clsUrls);
			}

			for (int i = classes.length - 1; i >= 1; i--) {
				if (result.get(classes[i]) != null)
					result.get(classes[i - 1]).addAll(result.get(classes[i]));
			}
		}

		return result;
	}

	private static void getWords(HashMap<String, HashSet<String>> allUrl,
			String host) {
		for (Entry<String, HashSet<String>> entry : allUrl.entrySet()) {
			ArrayList<Set> allSet = new ArrayList<Set>();
			TreeMap<String, Integer> allPair = new TreeMap<String, Integer>();

			for (String url : entry.getValue()) {
				allSet.add(getWordsLynx.runLynx(url));
				System.out.println("Getting page: " + url);
			}

			for (int m = 0; m < allSet.size(); m++) {
				for (Object word : allSet.get(m)) {
					if (!allPair.containsKey((String) word))
						allPair.put((String) word, 1);
					else
						allPair.put((String) word,
								allPair.get((String) word) + 1);
				}
			}

			writeFile(allPair, host, entry.getKey());
		}
	}

	private static void writeFile(TreeMap<String, Integer> allPair,
			String host, String category) {
		// TODO Auto-generated method stub
		try {
			FileWriter fstream = new FileWriter(category + "-" + host + ".txt");
			BufferedWriter out = new BufferedWriter(fstream);
			for (String word : allPair.keySet()) {
				out.write(word + "#" + allPair.get(word) + "\n");
			}

			out.close();
		} catch (Exception e) {// Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
	}

	public static void main(String[] args) throws IOException, JSONException {
		String host = args[0];
		int tec = Integer.parseInt(args[1]);
		double tes = Double.parseDouble(args[2]);

		ArrayList<String> result = ClassifyAlgorithm.getCategoryResult(host,
				tec, tes);

		HashMap<String, HashSet<String>> allurl = getAllUrl(host, result);
		getWords(allurl, host);
	}

}
